/*
 * @(#)ContentExtractor.java	1.0 05/01/07
 *
 * Copyright 2007 Fabio Gasparetti. All rights reserved.
 */

package org.tabularium.text.html;

import org.htmlparser.nodes.TagNode;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.HeadingTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.Span;
import org.htmlparser.tags.StyleTag;

/**
 * Helps extract content from Web pages.
 * 
 * @author Fabio Gasparetti
 * @version 1.0, 05/01/07
 */

public abstract class ContentExtractor {
	/* the following tags do not break sentences on their own;
	 * if there are spaces along with them, they will be included 
	 * in the output during parsing, breaking the flow.
	 * Note: maybe SPAN and DIV should be removed?
	 */
	public static final String[] DEFAULT_HTML_NOT_FLOWBREAK = { "STYLE", "SPAN",
			"LINK", "DIV", "H1", "H2", "H3", "H4", "H5", "H6", "STRONG", "B",
			"I", "U", "EM", "BIG", "SMALL", "TT", "SUP", "SUB", "A" /*, "P" */};

	protected String[] htmlNotFlowBreak = DEFAULT_HTML_NOT_FLOWBREAK;
	
	/**
	 * Extracts the textual content from the Web page.
	 * <p>
	 * Some implementations might require content freed of numeric character
	 * references and character entity
	 * 
	 * @see com.parc.search.commons.HtmlUtils#removeEscapedCh()
	 */
	public abstract String extractText(String url, String htmltext)
			throws Exception;

}
